# -*- coding: utf-8 -*-

from elasticsearch import Elasticsearch

import scpy.xawesome_time as xawsome_time

import util
from crawler import Crawler


class Newspaper(Crawler):

    def _crawl(self, params):
        retval = []

        eshosts = ['118.178.24.143']
        esbody = {
            'from': 0,
            'size': 100,
            'query': [
                {
                    'and': {
                        'bool': {
                            'should': [],
                            'must_not': [],
                            'must': [{
                                'match_phrase': {
                                    'content': {
                                        'query': params['keyword']
                                    }
                                }
                            }, {
                                'range': {
                                    'publishDate': {
                                        'gte': params['time_limit'],
                                        'lt': 'now/d'
                                    }
                                }
                            }]
                        }
                    }
                }
            ]
        }

        es = Elasticsearch(eshosts)

        res = es.search('newspaper', 'newspaper', body=esbody)
        data = [x.get('_source', {}) for x in res.get('hits', {}).get('hits', [])]
        if res.get('hits', {}).get('total', 0) >= 100:
            for i in range(100, res.get('hits', {}).get('total', 0), 100):
                esbody.update({'from': i})
                rs = es.search('newspaper', 'newspaper', esbody)
                data = data + [x.get('_source', {}) for x in rs.get('hits', {}).get('hits', [])]

        for x in data:
            try:
                content = x.get('content', '')
                retval.append({
                    "newsTime": xawsome_time.parse_time(res.get('publishDate', '')),
                    "crawlerSource": '报纸',
                    "sourceClassify": 'news',
                    "source": res.get('source', '报纸'),
                    "content": (content, '<p>{}</p>'.format(content)),
                    "title": res.get('title', ''),
                    "url": res.get('url', ''),
                    "abstract": res.get('content', '')[0:50],
                    "searchKey": params['keyword'],
                    "createdAt": util.current_date(),
                })
            except Exception as e:
                self._logger.warn('error: %s' % e)

        return retval
